package us.codecraft.webmagic.sample;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.lang3.StringUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.monitor.SpiderMonitor;
import us.codecraft.webmagic.pipeline.JsonFilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import javax.management.JMException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

/**
 * 微信公众号文章爬虫规则
 *
 * @Author: xu.pengfei
 * @Email: faberxu@gmail.com
 * @Date: 2018-08-08 14:11
 */
public class WeixinPostProcessor implements PageProcessor {

    private List<String> strCacheList;

    /**
     * 抓取网站的相关配置，包括编码、抓取间隔、重试次数等
     */
    private Site site = Site
            .me()
            .setDomain("mp.weixin.qq.com")
            .setSleepTime(3000)
            .setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");

    @Override
    public void process(Page page) {
        strCacheList = new ArrayList<String>();
        JSONObject json = new JSONObject();

        // 解析文章作者
        String author = page.getHtml().xpath("//span[@class='rich_media_meta rich_media_meta_text']//text()").get();
        json.put("author", author);

        // 解析文章主体内容
        Selectable content = page.getHtml().xpath("//div[@id='js_content']");
        JSONArray list = new JSONArray();
        processSection(page, content, list);
        System.out.println(list.toString());
    }

    private JSONArray processSection(Page page, Selectable content, JSONArray list) {
        List<Selectable> childList = content.xpath("/*/*").nodes();
        for (Selectable child : childList) {
            // 如果是链接<a href/>标签，节点以下内容解析为a标签内部内容
            if (child.xpath("/a").get() != null) {
//                System.out.println("--------------->>>>> parse a href begin ------------->>>>");
                String link = child.xpath("//a//@href").get();
                JSONObject data = printStr(link, "link");
                if (data != null) {
                    JSONArray linkDataList = new JSONArray();
                    processSection(page, child, linkDataList);
                    data.put("linkContent", linkDataList);
                    list.add(data);
                }

//                System.out.println("--------------->>>>> parse a href end ------------->>>>");
                continue;
            }

            processSection(page, child, list);


//            System.out.println(child.get());
            // 图片
            List<Selectable> imgList = child.xpath("//img").nodes();
            if (imgList != null && !imgList.isEmpty()) {
                for (Selectable img : imgList) {
                    String imgUrl = img.xpath("//*//@data-src").get();
                    JSONObject data = printStr(imgUrl, "img");
                    if (data != null) {
                        list.add(data);
                    }
                }
            }

            // 视频
            List<Selectable> videoList = child.xpath("//iframe").nodes();
            if (videoList != null && !videoList.isEmpty()) {
                for (Selectable video : videoList) {
                    String videoUrl = video.xpath("//*//@data-src").get();
                    JSONObject data = printStr(videoUrl, "video");
                    if (data != null) {
                        list.add(data);
                    }
                }
            }

            // 文本
            String text = child.xpath("/*/text()").get();
            if (StringUtils.isNotEmpty(text)) {
                JSONObject data = printStr(text, "text");
                if (data != null) {
                    list.add(data);
                }
            }
        }

        return list;
    }

    private JSONObject printStr(String str, String label) {
        if (strCacheList.contains(str)) {
            return null;
        }
        strCacheList.add(str);
//        System.out.println("[" + label + "]:" + str);

        JSONObject data = new JSONObject();
        data.put("type", label);
        data.put("value", str);
        return data;
    }

    @Override
    public Site getSite() {
        return site;
    }

    public static void startSpider(List<WeixinPost> inposts) throws JMException {
        long startTime, endTime;
        startTime = System.currentTimeMillis();

//        List<String> urls = new ArrayList<>();
        List<Request> requests = new ArrayList<Request>();
        for (WeixinPost weixinPost : inposts) {
//            urls.add(weixinPost.getContentUrl());

            Request request = new Request(weixinPost.getContentUrl());
            request.putExtra("post", weixinPost);
            requests.add(request);
        }

        Request[] rq = requests.toArray(new Request[]{});
        Spider spider = Spider.create(new WeixinPostProcessor())
                .addRequest(rq)
//                .addUrl("http://mp.weixin.qq.com/s?__biz=MzI3MzAzMjMwNQ==&mid=2653785776&idx=1&sn=dd4b815ccf99954fef3d7758923d95f1&chksm=f0f05baec787d2b8baf7f0c2702aba3fcc2c12db99a5b66ea232a22fc49dc004eca0caba29ec&scene=27#wechat_redirect")
                .addPipeline(new JsonFilePipeline("D:\\webmagic"))
                .thread(5);

        SpiderMonitor.instance().register(spider);

        spider.start();
    }

    public static void main(String[] args) throws JMException {
        String url = "http://mp.weixin.qq.com/s?__biz=MzI3MzAzMjMwNQ==&mid=2653785776&idx=1&sn=dd4b815ccf99954fef3d7758923d95f1&chksm=f0f05baec787d2b8baf7f0c2702aba3fcc2c12db99a5b66ea232a22fc49dc004eca0caba29ec&scene=27#wechat_redirect";
        // String url = "https://mp.weixin.qq.com/s?__biz=MjM5MjcyNzU1OA==&mid=2651121571&idx=1&sn=681c694429e3137ec5e981aa94b6d19b&chksm=bd5190ca8a2619dc5b45a3049ce36e98d824d3b67422a468f86cab000aceb70c5e038fe3922d&scene=27#wechat_redirect";
        // String url = "http://mp.weixin.qq.com/s?__biz=MjM5MjcyNzU1OA==&mid=2651121463&idx=2&sn=4bf1d654761c7c6d075ce1fa3417e193&chksm=bd51905e8a261948a4f65b6511fdd60cc228c891fe47857703709ebd9c215980ef93fe6f40c1&scene=27#wechat_redirect";
        // String url = "http://mp.weixin.qq.com/s?__biz=MjM5OTY4NjAwMQ==&mid=2650094232&idx=4&sn=c312b5ff360e2c021a78d08e002afbe5&chksm=bf3620868841a9906ea9a4ec9c9bfb98f70bf18f2a8d02acb4c51e6b98be90f80550aa423f62&scene=27#wechat_redirect";

        WeixinPost post = new WeixinPost();
        post.setContentUrl(url);
        startSpider(Arrays.asList(post));
    }

}
